package ca.uwindsor.cs.deepweb.utility.documentanalysis;

import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;

import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.TermEnum;

public class TermsAndFrequencyExtractorByRange {

	public static void main(String[] args) {

		if (args.length != 2) {
			System.out
					.println("java -jar TermsAndFrequencyExtractor.jar indexDir outputFile");
			System.exit(0);
		}
		try {
			IndexReader reader = IndexReader.open(args[0]);
			System.out.println("NumDocs = " + reader.numDocs());
			// HashMap<String, Integer> wordlist = new HashMap<String,
			// Integer>(500000);
			// // HashMap<String, Integer> digitlist = new HashMap<String,
			// // Integer>();
			// int i, j, num;
			// // Pattern patterndigit = Pattern.compile("\\d+");
			// // Pattern patternletter = Pattern.compile("[a-z]+");
			// // Pattern patternpurelynumbers = Pattern.compile("^(\\d+,*)+$");
			// // Pattern patternpurelyletters =
			// // Pattern.compile("^([A-Za-z]+,*)+$");
			// // Matcher map;
			// Integer v;
			// String line;
			// TermFreqVector termFreqVector;
			// String[] terms;
			// String word;
			// int[] frequences;
			// num = reader.numDocs();
			// for (i = 0; i < num; i++) {
			// // 对contents的token之后的term存于了TermFreqVector
			// termFreqVector = reader.getTermFreqVector(i,
			// FilesContentIndexer.FIELD_CONTENT);
			//
			// if (termFreqVector == null) {
			// System.out.println("termFreqVector is null.");
			// continue;
			// }
			//
			// // String fieldName = termFreqVector.getField();
			// terms = termFreqVector.getTerms();
			// frequences = termFreqVector.getTermFrequencies();
			//
			// // System.out.println("FieldName:" + fieldName);
			// for (j = 0; j < terms.length; j++) {
			// word = terms[j];
			// // mad = patterndigit.matcher(word);
			// // mac = patternletter.matcher(word);
			// // map = patternpurelynumbers.matcher(word);
			// // if (!map.find()) {// find
			// v = wordlist.get(word);
			// if (v == null) {
			// wordlist.put(word, 1);
			// } else {
			// v = v.intValue() + frequences[j];
			// wordlist.put(word, v);
			// }
			// // }
			// }
			// if (i % 10000 == 0) {
			// System.gc();
			// System.out.println(i + " documents are processed!");
			// }
			// }
			// //
			// FileOutputStream fp = new FileOutputStream(args[1]);
			// Set<Entry<String, Integer>> s = wordlist.entrySet();
			// for (Entry<String, Integer> e : s) {
			// line = e.getKey() + "\t" + e.getValue() + "\n";
			// fp.write(line.getBytes("UTF-8"));
			// }
			// fp.flush();
			// fp.close();
			// //
			// // fp = new FileOutputStream(args[2]);
			// // s = digitlist.entrySet();
			// // for (Entry<String, Integer> e : s) {
			// // line = e.getKey() + "\t" + e.getValue() + "\n";
			// // fp.write(line.getBytes("UTF-8"));
			// // }
			// // fp.flush();
			// // fp.close();
			ArrayList<Integer> list = new ArrayList<Integer>();

			// Pattern p = Pattern.compile("[,\\d]+");
			// Matcher matcher;
			TermEnum te = reader.terms();
			while (te.next()) {
				// matcher = p.matcher(te.term().text());
				// if (!matcher.find()) {
				// TermFrequencyCell cell = new
				// TermFrequencyCell(te.term().text(),te.docFreq());
				// line = te.term().text() + "\t" + te.docFreq() + "\n";
				// fp.write(line.getBytes("UTF-8"));
				if (te.docFreq() > 10) {
					te.term().text();
					list.add(te.docFreq());
					// }
				}
			}
			te.close();
			reader.close();
			System.gc();
			Collections.sort(list);
			Collections.reverse(list);

			FileOutputStream fpy = new FileOutputStream(args[1]);
			for (int i = 0; i < list.size(); i++) {
				Integer cell = list.get(i);

				fpy.write(Integer.toString(cell.intValue()).getBytes());
				fpy.write(" ".getBytes());
				fpy.flush();
				// line = Math.log10(i+1) + "\t" + Math.log10(cell.intValue()) +
				// "\n";

			}
			fpy.close();
		} catch (CorruptIndexException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}

}
